import os
from datetime import datetime
from DouYuTvSpider import DouYuTvSpider
from multiprocessing import Pool, cpu_count, Manager


# 多进程执行
class multiple_processes_main(object):
    def __init__(self):
        self.douyutvspider = DouYuTvSpider()
        self.cpu_count = cpu_count()

    # 获取分页列表
    def get_page_list(self):
        page = int(self.douyutvspider.page()[0])
        # 比如cpu是双核的，则返回2，如果双四核cpu，则返回8
        size = int(page / self.cpu_count)
        page_list = []
        num_list = []
        j = 1
        k = 1
        for i in range(0, page):
            if k == cpu_count:
                num_list.append(i + 1)
                if i + 1 == page:
                    page_list.append(num_list)
                    continue

            if j == size:
                j = 1
                k += 1
                page_list.append(num_list)
                num_list = []

            num_list.append(i + 1)
            j += 1
        return page_list

    def run(self, num_list, analysis_info_list):
        print("进程id", os.getpid())
        self.douyutvspider.go(num_list, analysis_info_list)


if __name__ == '__main__':
    manager = Manager()
    analysis_info_list = manager.list()
    main = multiple_processes_main()
    p = Pool(main.cpu_count)
    start_time = datetime.now()

    for num_list in main.get_page_list():
        p.apply_async(main.run, args=(num_list, analysis_info_list,))
    print('Waiting for all subprocesses done...')
    p.close()
    p.join()
    main.douyutvspider.show(
        main.douyutvspider.sort(analysis_info_list))
    print("耗时", (datetime.now() - start_time).seconds, "秒")
